{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7d0c0193",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8dd009a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/conda/envs/dpf/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "100%|██████████| 1/1 [00:00<00:00, 284.96it/s]\n"
     ]
    }
   ],
   "source": [
    "from DPF import ShardedFilesDatasetConfig, DatasetReader\n",
    "\n",
    "config = ShardedFilesDatasetConfig.from_path_and_columns(\n",
    "    'example_video_dataset',\n",
    "    video_name_col='video_name',\n",
    "    text_col='caption'\n",
    ")\n",
    "\n",
    "reader = DatasetReader()\n",
    "processor = reader.read_from_config(config)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad82425c",
   "metadata": {},
   "source": [
    "# Check dataset and it's info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6427374f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset format: sharded_files\n",
      "Path: example_video_dataset\n",
      "Modalities: ['video', 'text']\n",
      "Columns: 3\n",
      "Total samples: 5\n"
     ]
    }
   ],
   "source": [
    "processor.print_summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "943f1b5a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>video_path</th>\n",
       "      <th>text</th>\n",
       "      <th>split_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>example_video_dataset/0/0.mp4</td>\n",
       "      <td>Businessman Rides In A Car</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>example_video_dataset/0/1.mp4</td>\n",
       "      <td>Workspace In Natural Light</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>example_video_dataset/0/2.mp4</td>\n",
       "      <td>Woman In Dress Walking In Tulip Field</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>example_video_dataset/0/3.mp4</td>\n",
       "      <td>Film Burns Overlay</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>example_video_dataset/0/4.mp4</td>\n",
       "      <td>Portrait Leader Of The Roman Army</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      video_path                                   text  \\\n",
       "0  example_video_dataset/0/0.mp4             Businessman Rides In A Car   \n",
       "1  example_video_dataset/0/1.mp4             Workspace In Natural Light   \n",
       "2  example_video_dataset/0/2.mp4  Woman In Dress Walking In Tulip Field   \n",
       "3  example_video_dataset/0/3.mp4                     Film Burns Overlay   \n",
       "4  example_video_dataset/0/4.mp4      Portrait Leader Of The Roman Army   \n",
       "\n",
       "  split_name  \n",
       "0          0  \n",
       "1          0  \n",
       "2          0  \n",
       "3          0  \n",
       "4          0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processor.df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2122543",
   "metadata": {},
   "source": [
    "# Running filters"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7bef595a",
   "metadata": {},
   "source": [
    "## VideoInfoFilter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b2fa0081",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['is_correct', 'error', 'width', 'height', 'fps', 'duration']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [00:00<00:00, 15.06it/s]\n"
     ]
    }
   ],
   "source": [
    "from DPF.filters.videos.info_filter import VideoInfoFilter\n",
    "\n",
    "datafilter = VideoInfoFilter(workers=16) \n",
    "print(datafilter.result_columns) # prints list of colums that will be added\n",
    "\n",
    "processor.apply_data_filter(datafilter)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b6599f38",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>video_path</th>\n",
       "      <th>text</th>\n",
       "      <th>split_name</th>\n",
       "      <th>is_correct</th>\n",
       "      <th>error</th>\n",
       "      <th>width</th>\n",
       "      <th>height</th>\n",
       "      <th>fps</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>example_video_dataset/0/0.mp4</td>\n",
       "      <td>Businessman Rides In A Car</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>14.2800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>example_video_dataset/0/1.mp4</td>\n",
       "      <td>Workspace In Natural Light</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>10.0800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>example_video_dataset/0/2.mp4</td>\n",
       "      <td>Woman In Dress Walking In Tulip Field</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>12.1200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>example_video_dataset/0/3.mp4</td>\n",
       "      <td>Film Burns Overlay</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>23.976024</td>\n",
       "      <td>24.5245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>example_video_dataset/0/4.mp4</td>\n",
       "      <td>Portrait Leader Of The Roman Army</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>29.970030</td>\n",
       "      <td>9.2092</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      video_path                                   text  \\\n",
       "0  example_video_dataset/0/0.mp4             Businessman Rides In A Car   \n",
       "1  example_video_dataset/0/1.mp4             Workspace In Natural Light   \n",
       "2  example_video_dataset/0/2.mp4  Woman In Dress Walking In Tulip Field   \n",
       "3  example_video_dataset/0/3.mp4                     Film Burns Overlay   \n",
       "4  example_video_dataset/0/4.mp4      Portrait Leader Of The Roman Army   \n",
       "\n",
       "  split_name  is_correct error  width  height        fps  duration  \n",
       "0          0        True  None   1280     720  25.000000   14.2800  \n",
       "1          0        True  None   1280     720  25.000000   10.0800  \n",
       "2          0        True  None   1280     720  25.000000   12.1200  \n",
       "3          0        True  None   1280     720  23.976024   24.5245  \n",
       "4          0        True  None   1280     720  29.970030    9.2092  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processor.df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b179d86",
   "metadata": {},
   "source": [
    "## Farneback optical flow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c0d35f05",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [00:03<00:00,  1.38it/s]\n"
     ]
    }
   ],
   "source": [
    "from DPF.filters.videos.farneback_filter import GunnarFarnebackFilter\n",
    "\n",
    "datafilter = GunnarFarnebackFilter(pass_frames=12, min_frame_size=512, num_passes=12, workers=8)\n",
    "processor.apply_data_filter(datafilter)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "76686c64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    4.601\n",
       "1    0.832\n",
       "2    7.847\n",
       "3    0.018\n",
       "4    3.187\n",
       "Name: optical_flow_farneback, dtype: float32"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processor.df['optical_flow_farneback']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85a57c57",
   "metadata": {},
   "source": [
    "## RAFT optical flow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1b4f9fcf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [00:04<00:00,  1.07it/s]\n"
     ]
    }
   ],
   "source": [
    "from DPF.filters.videos.raft_filter import RAFTOpticalFlowFilter\n",
    "\n",
    "datafilter = RAFTOpticalFlowFilter(pass_frames=12, min_frame_size=512, num_passes=12, workers=8)\n",
    "processor.apply_data_filter(datafilter)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "759297a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     8.898000\n",
       "1     4.232000\n",
       "2    19.886000\n",
       "3    41.337002\n",
       "4    25.343000\n",
       "Name: optical_flow_raft, dtype: float32"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processor.df['optical_flow_raft']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d96ead27",
   "metadata": {},
   "source": [
    "## VideoLLaVA captioning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "37a59d40",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2024-03-29 15:16:16,768] [INFO] [real_accelerator.py:161:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jovyan/.mlspace/envs/llava_34b/lib/python3.11/site-packages/torchvision/transforms/_functional_video.py:6: UserWarning: The 'torchvision.transforms._functional_video' module is deprecated since 0.12 and will be removed in the future. Please use the 'torchvision.transforms.functional' module instead.\n",
      "  warnings.warn(\n",
      "/home/jovyan/.mlspace/envs/llava_34b/lib/python3.11/site-packages/torchvision/transforms/_transforms_video.py:22: UserWarning: The 'torchvision.transforms._transforms_video' module is deprecated since 0.12 and will be removed in the future. Please use the 'torchvision.transforms' module instead.\n",
      "  warnings.warn(\n",
      "/home/jovyan/.mlspace/envs/llava_34b/lib/python3.11/site-packages/torchvision/transforms/functional_tensor.py:5: UserWarning: The torchvision.transforms.functional_tensor module is deprecated in 0.15 and will be **removed in 0.17**. Please don't rely on it. You probably just need to use APIs in torchvision.transforms.functional or in torchvision.transforms.v2.functional.\n",
      "  warnings.warn(\n",
      "config.json: 100%|██████████| 1.09k/1.09k [00:00<00:00, 4.03MB/s]\n",
      "Downloading shards: 100%|██████████| 2/2 [00:00<00:00,  5.66it/s]\n",
      "Loading checkpoint shards: 100%|██████████| 2/2 [00:34<00:00, 17.04s/it]\n",
      "Some weights of the model checkpoint at LanguageBind/Video-LLaVA-7B were not used when initializing LlavaLlamaForCausalLM: ['model.image_tower.image_tower.encoder.layers.16.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.16.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.10.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.3.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.20.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.5.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.4.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.15.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.23.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.11.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.20.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.13.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.22.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.3.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.15.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.6.temporal_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.14.mlp.fc2.weight', 'model.image_tower.image_tower.encoder.layers.15.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.12.self_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.4.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.18.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.7.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.16.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.20.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.1.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.8.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.18.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.6.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.5.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.0.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.17.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.8.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.17.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.17.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.21.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.19.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.11.self_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.18.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.14.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.16.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.13.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.1.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.12.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.19.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.11.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.8.temporal_embedding', 'model.image_tower.image_tower.encoder.layers.23.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.14.temporal_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.15.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.21.temporal_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.4.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.15.temporal_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.7.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.15.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.21.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.2.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.7.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.20.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.12.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.0.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.9.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.4.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.21.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.17.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.4.mlp.fc2.weight', 'model.image_tower.image_tower.encoder.layers.21.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.6.self_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.9.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.3.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.23.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.20.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.6.temporal_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.2.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.5.temporal_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.6.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.14.temporal_layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.2.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.19.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.1.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.8.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.12.temporal_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.22.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.16.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.16.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.21.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.1.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.2.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.4.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.7.temporal_layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.7.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.10.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.10.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.2.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.21.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.10.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.13.temporal_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.2.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.5.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.1.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.3.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.19.temporal_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.14.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.2.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.8.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.2.mlp.fc2.weight', 'model.image_tower.image_tower.encoder.layers.21.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.14.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.13.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.11.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.18.self_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.22.self_attn.v_proj.weight', 'model.video_tower.video_tower.embeddings.position_embedding.weight', 'model.image_tower.image_tower.encoder.layers.17.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.14.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.23.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.5.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.3.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.23.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.21.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.9.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.19.temporal_embedding', 'model.video_tower.video_tower.encoder.layers.12.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.post_layernorm.bias', 'model.image_tower.image_tower.encoder.layers.17.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.12.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.15.temporal_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.13.temporal_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.13.self_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.17.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.11.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.23.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.14.temporal_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.7.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.8.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.20.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.7.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.17.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.12.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.20.mlp.fc1.bias', 'model.image_tower.image_tower.encoder.layers.18.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.5.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.22.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.14.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.8.self_attn.out_proj.weight', 'model.image_tower.image_tower.pre_layrnorm.bias', 'model.image_tower.image_tower.encoder.layers.6.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.8.self_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.0.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.12.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.7.temporal_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.13.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.2.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.14.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.21.temporal_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.3.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.0.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.6.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.1.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.4.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.19.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.2.temporal_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.21.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.5.temporal_layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.14.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.11.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.14.mlp.fc2.weight', 'model.image_tower.image_tower.encoder.layers.22.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.17.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.15.temporal_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.20.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.23.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.10.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.6.temporal_embedding', 'model.image_tower.image_tower.encoder.layers.11.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.3.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.22.temporal_attn.k_proj.weight', 'model.image_tower.image_tower.embeddings.position_embedding.weight', 'model.video_tower.video_tower.encoder.layers.20.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.22.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.3.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.4.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.14.temporal_layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.23.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.9.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.8.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.21.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.7.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.16.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.17.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.8.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.11.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.4.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.8.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.4.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.22.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.4.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.11.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.2.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.21.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.16.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.2.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.4.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.16.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.11.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.16.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.23.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.13.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.17.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.5.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.7.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.12.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.23.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.15.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.18.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.4.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.22.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.12.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.11.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.0.temporal_embedding', 'model.video_tower.video_tower.encoder.layers.4.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.0.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.2.self_attn.out_proj.bias', 'model.image_tower.image_tower.post_layernorm.bias', 'model.video_tower.video_tower.encoder.layers.4.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.15.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.5.temporal_layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.0.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.15.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.3.temporal_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.13.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.9.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.22.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.10.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.3.temporal_embedding', 'model.video_tower.video_tower.encoder.layers.22.temporal_layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.3.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.21.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.7.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.10.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.16.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.18.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.4.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.9.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.10.temporal_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.1.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.8.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.7.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.4.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.2.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.12.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.8.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.4.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.14.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.2.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.6.temporal_layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.8.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.9.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.13.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.14.temporal_embedding', 'model.image_tower.image_tower.encoder.layers.13.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.18.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.7.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.22.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.14.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.20.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.2.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.6.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.9.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.23.self_attn.k_proj.weight', 'model.video_tower.video_tower.embeddings.class_embedding', 'model.image_tower.image_tower.encoder.layers.17.mlp.fc1.bias', 'model.image_tower.image_tower.encoder.layers.7.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.5.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.15.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.0.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.23.temporal_layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.14.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.10.temporal_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.19.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.16.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.15.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.19.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.11.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.5.temporal_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.10.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.7.temporal_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.21.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.12.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.7.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.0.temporal_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.23.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.9.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.14.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.5.mlp.fc1.bias', 'model.image_tower.image_tower.encoder.layers.23.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.9.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.9.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.17.temporal_embedding', 'model.image_tower.image_tower.encoder.layers.0.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.20.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.13.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.19.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.20.temporal_embedding', 'model.video_tower.video_tower.encoder.layers.17.temporal_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.2.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.6.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.14.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.5.temporal_embedding', 'model.image_tower.image_tower.encoder.layers.2.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.10.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.21.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.22.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.11.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.20.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.17.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.22.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.15.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.19.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.17.self_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.8.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.3.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.6.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.18.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.9.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.12.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.9.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.4.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.3.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.12.temporal_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.7.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.19.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.22.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.5.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.18.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.16.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.0.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.7.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.5.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.7.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.3.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.17.temporal_layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.4.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.16.temporal_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.1.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.12.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.13.temporal_embedding', 'model.video_tower.video_tower.encoder.layers.14.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.9.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.16.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.23.temporal_embedding', 'model.image_tower.image_tower.encoder.layers.14.mlp.fc1.bias', 'model.image_tower.image_tower.encoder.layers.18.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.19.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.4.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.2.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.21.temporal_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.19.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.3.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.3.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.18.layer_norm1.bias', 'model.video_tower.video_tower.embeddings.patch_embedding.weight', 'model.video_tower.video_tower.encoder.layers.23.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.2.temporal_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.2.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.12.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.20.temporal_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.6.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.5.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.4.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.11.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.16.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.4.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.19.temporal_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.19.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.20.temporal_layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.12.temporal_layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.15.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.19.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.15.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.23.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.1.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.12.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.18.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.8.mlp.fc2.weight', 'model.image_tower.image_tower.encoder.layers.9.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.1.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.9.temporal_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.5.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.17.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.18.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.14.temporal_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.20.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.20.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.20.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.1.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.11.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.5.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.16.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.10.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.14.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.0.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.23.temporal_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.7.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.14.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.7.temporal_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.1.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.10.temporal_layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.6.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.0.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.15.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.20.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.1.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.13.self_attn.out_proj.weight', 'model.image_tower.image_tower.post_layernorm.weight', 'model.image_tower.image_tower.encoder.layers.13.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.12.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.14.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.14.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.6.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.3.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.2.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.2.temporal_layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.19.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.21.temporal_layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.19.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.8.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.18.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.0.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.17.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.3.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.15.temporal_embedding', 'model.image_tower.image_tower.encoder.layers.3.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.9.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.13.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.1.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.18.temporal_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.8.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.11.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.6.temporal_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.14.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.0.temporal_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.1.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.1.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.8.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.9.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.22.temporal_layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.11.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.10.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.23.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.6.mlp.fc1.bias', 'model.image_tower.image_tower.encoder.layers.15.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.15.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.16.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.4.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.0.temporal_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.0.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.16.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.11.temporal_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.3.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.7.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.1.mlp.fc1.bias', 'model.image_tower.image_tower.encoder.layers.20.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.22.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.11.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.2.temporal_layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.9.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.23.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.11.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.7.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.8.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.18.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.3.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.1.self_attn.v_proj.bias', 'model.video_tower.video_tower.pre_layrnorm.weight', 'model.image_tower.image_tower.encoder.layers.21.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.21.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.13.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.3.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.18.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.10.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.3.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.16.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.3.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.6.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.12.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.19.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.8.temporal_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.5.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.6.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.16.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.12.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.21.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.0.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.19.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.8.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.21.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.1.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.0.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.15.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.22.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.7.mlp.fc2.weight', 'model.image_tower.image_tower.encoder.layers.15.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.12.mlp.fc2.weight', 'model.image_tower.image_tower.encoder.layers.20.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.20.temporal_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.3.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.1.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.4.temporal_embedding', 'model.video_tower.video_tower.encoder.layers.2.temporal_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.0.temporal_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.7.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.4.temporal_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.4.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.14.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.7.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.2.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.5.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.12.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.10.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.13.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.4.temporal_layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.17.self_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.2.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.17.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.1.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.8.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.11.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.0.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.21.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.12.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.3.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.17.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.17.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.22.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.22.temporal_embedding', 'model.image_tower.image_tower.encoder.layers.7.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.17.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.11.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.23.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.3.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.23.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.11.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.19.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.11.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.0.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.4.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.12.temporal_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.18.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.3.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.19.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.10.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.23.temporal_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.16.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.10.temporal_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.9.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.21.temporal_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.12.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.19.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.20.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.16.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.7.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.11.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.19.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.3.temporal_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.14.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.2.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.21.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.3.temporal_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.18.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.1.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.1.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.10.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.11.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.10.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.7.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.8.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.10.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.16.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.21.mlp.fc2.weight', 'model.image_tower.image_tower.encoder.layers.14.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.15.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.10.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.20.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.16.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.16.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.2.temporal_attn.q_proj.bias', 'model.image_tower.image_tower.pre_layrnorm.weight', 'model.video_tower.video_tower.encoder.layers.23.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.13.self_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.12.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.12.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.8.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.15.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.22.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.0.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.6.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.9.temporal_layer_norm1.weight', 'model.image_tower.image_tower.embeddings.patch_embedding.weight', 'model.video_tower.video_tower.encoder.layers.18.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.9.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.22.temporal_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.0.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.22.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.4.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.2.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.6.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.8.temporal_layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.12.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.19.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.6.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.0.temporal_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.1.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.9.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.22.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.18.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.2.temporal_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.19.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.15.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.16.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.11.self_attn.k_proj.weight', 'model.image_tower.image_tower.embeddings.class_embedding', 'model.video_tower.video_tower.encoder.layers.10.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.7.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.14.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.4.temporal_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.17.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.2.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.1.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.18.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.11.temporal_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.23.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.9.temporal_embedding', 'model.image_tower.image_tower.encoder.layers.7.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.1.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.6.temporal_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.11.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.18.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.7.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.2.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.21.temporal_embedding', 'model.video_tower.video_tower.encoder.layers.7.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.11.temporal_embedding', 'model.image_tower.image_tower.encoder.layers.4.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.3.temporal_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.15.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.15.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.22.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.9.temporal_layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.13.temporal_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.18.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.5.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.8.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.20.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.5.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.3.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.15.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.18.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.19.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.23.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.13.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.17.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.5.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.6.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.13.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.14.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.0.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.17.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.17.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.8.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.22.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.13.temporal_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.1.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.23.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.16.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.17.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.20.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.11.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.23.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.13.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.5.self_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.4.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.11.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.13.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.20.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.0.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.22.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.12.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.23.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.0.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.22.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.10.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.0.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.18.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.21.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.5.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.18.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.17.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.0.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.13.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.20.mlp.fc2.weight', 'model.image_tower.image_tower.encoder.layers.6.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.9.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.21.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.0.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.19.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.17.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.6.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.4.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.5.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.20.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.20.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.5.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.22.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.21.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.4.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.3.temporal_layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.21.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.14.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.22.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.11.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.5.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.23.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.1.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.18.temporal_layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.10.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.10.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.6.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.0.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.15.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.19.temporal_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.21.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.13.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.13.temporal_layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.10.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.12.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.6.temporal_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.19.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.0.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.21.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.10.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.20.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.12.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.9.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.17.temporal_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.14.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.13.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.19.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.15.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.2.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.8.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.18.temporal_embedding', 'model.image_tower.image_tower.encoder.layers.23.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.1.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.2.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.6.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.1.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.6.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.8.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.16.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.7.temporal_embedding', 'model.image_tower.image_tower.encoder.layers.10.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.12.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.17.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.10.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.11.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.19.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.21.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.15.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.18.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.16.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.7.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.23.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.21.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.18.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.8.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.0.temporal_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.18.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.20.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.16.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.14.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.13.temporal_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.5.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.14.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.1.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.15.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.3.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.16.temporal_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.0.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.9.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.7.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.5.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.7.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.23.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.6.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.8.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.2.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.18.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.1.temporal_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.13.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.14.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.22.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.15.temporal_layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.11.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.16.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.22.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.17.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.6.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.9.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.17.temporal_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.16.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.14.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.4.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.15.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.18.temporal_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.22.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.1.temporal_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.10.mlp.fc2.weight', 'model.image_tower.image_tower.encoder.layers.7.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.15.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.21.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.2.temporal_embedding', 'model.video_tower.video_tower.encoder.layers.1.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.post_layernorm.weight', 'model.image_tower.image_tower.encoder.layers.0.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.6.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.20.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.3.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.15.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.16.temporal_embedding', 'model.video_tower.video_tower.encoder.layers.8.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.10.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.13.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.8.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.4.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.17.temporal_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.5.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.12.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.6.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.9.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.19.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.0.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.4.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.13.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.20.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.3.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.1.temporal_layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.9.temporal_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.7.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.5.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.9.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.16.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.11.temporal_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.9.temporal_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.9.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.23.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.9.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.18.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.23.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.19.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.14.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.10.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.6.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.6.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.6.temporal_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.2.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.16.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.16.temporal_layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.6.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.1.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.10.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.0.temporal_layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.12.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.1.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.5.temporal_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.19.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.7.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.15.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.18.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.15.self_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.5.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.5.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.14.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.20.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.3.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.0.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.2.temporal_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.10.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.19.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.19.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.20.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.11.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.0.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.13.mlp.fc2.weight', 'model.image_tower.image_tower.encoder.layers.3.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.1.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.9.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.8.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.18.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.6.temporal_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.12.mlp.fc1.bias', 'model.image_tower.image_tower.encoder.layers.23.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.10.temporal_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.20.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.10.temporal_embedding', 'model.video_tower.video_tower.encoder.layers.23.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.5.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.3.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.16.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.9.temporal_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.22.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.3.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.5.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.7.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.21.temporal_layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.2.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.19.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.8.temporal_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.20.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.9.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.16.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.5.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.9.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.11.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.17.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.6.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.21.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.22.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.14.temporal_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.17.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.10.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.23.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.15.layer_norm2.weight', 'model.video_tower.video_tower.encoder.layers.2.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.21.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.15.temporal_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.18.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.13.temporal_layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.10.mlp.fc1.bias', 'model.image_tower.image_tower.encoder.layers.10.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.15.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.8.self_attn.out_proj.bias', 'model.image_tower.image_tower.encoder.layers.8.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.23.self_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.12.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.8.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.19.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.22.temporal_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.21.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.13.mlp.fc1.bias', 'model.image_tower.image_tower.encoder.layers.19.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.12.temporal_embedding', 'model.image_tower.image_tower.encoder.layers.18.mlp.fc2.weight', 'model.video_tower.video_tower.pre_layrnorm.bias', 'model.image_tower.image_tower.encoder.layers.9.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.20.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.0.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.23.temporal_attn.q_proj.bias', 'model.image_tower.image_tower.encoder.layers.8.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.20.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.11.temporal_layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.5.mlp.fc2.weight', 'model.image_tower.image_tower.encoder.layers.5.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.22.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.19.mlp.fc1.bias', 'model.image_tower.image_tower.encoder.layers.1.mlp.fc2.weight', 'model.image_tower.image_tower.encoder.layers.3.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.12.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.22.mlp.fc2.weight', 'model.video_tower.video_tower.encoder.layers.7.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.2.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.13.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.9.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.12.self_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.22.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.5.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.4.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.13.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.4.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.17.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.16.temporal_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.17.temporal_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.21.temporal_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.19.mlp.fc1.weight', 'model.image_tower.image_tower.encoder.layers.12.self_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.9.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.18.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.22.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.1.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.16.self_attn.out_proj.weight', 'model.image_tower.image_tower.encoder.layers.10.mlp.fc2.bias', 'model.image_tower.image_tower.encoder.layers.20.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.1.temporal_embedding', 'model.video_tower.video_tower.encoder.layers.23.layer_norm2.bias', 'model.video_tower.video_tower.encoder.layers.13.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.21.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.21.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.15.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.9.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.12.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.1.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.4.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.6.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.11.mlp.fc2.weight', 'model.image_tower.image_tower.encoder.layers.18.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.13.mlp.fc1.bias', 'model.image_tower.image_tower.encoder.layers.11.self_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.13.self_attn.k_proj.weight', 'model.video_tower.video_tower.encoder.layers.11.temporal_attn.k_proj.bias', 'model.video_tower.video_tower.encoder.layers.13.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.14.layer_norm2.weight', 'model.image_tower.image_tower.encoder.layers.11.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.0.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.19.temporal_layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.15.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.4.temporal_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.6.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.23.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.18.layer_norm1.weight', 'model.video_tower.video_tower.encoder.layers.12.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.20.temporal_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.23.layer_norm1.bias', 'model.image_tower.image_tower.encoder.layers.14.self_attn.out_proj.bias', 'model.video_tower.video_tower.encoder.layers.22.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.3.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.10.self_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.16.mlp.fc1.bias', 'model.video_tower.video_tower.encoder.layers.8.temporal_attn.out_proj.weight', 'model.video_tower.video_tower.encoder.layers.17.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.22.layer_norm1.weight', 'model.image_tower.image_tower.encoder.layers.2.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.3.mlp.fc2.bias', 'model.video_tower.video_tower.encoder.layers.5.temporal_attn.q_proj.weight', 'model.video_tower.video_tower.encoder.layers.20.temporal_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.10.self_attn.v_proj.weight', 'model.video_tower.video_tower.encoder.layers.8.layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.8.temporal_layer_norm1.bias', 'model.video_tower.video_tower.encoder.layers.0.temporal_attn.v_proj.bias', 'model.image_tower.image_tower.encoder.layers.1.self_attn.q_proj.bias', 'model.video_tower.video_tower.encoder.layers.22.temporal_attn.v_proj.weight', 'model.image_tower.image_tower.encoder.layers.6.layer_norm2.bias', 'model.image_tower.image_tower.encoder.layers.5.self_attn.v_proj.bias', 'model.video_tower.video_tower.encoder.layers.13.mlp.fc1.weight', 'model.video_tower.video_tower.encoder.layers.4.self_attn.k_proj.bias', 'model.image_tower.image_tower.encoder.layers.17.self_attn.q_proj.weight', 'model.image_tower.image_tower.encoder.layers.17.self_attn.k_proj.weight', 'model.image_tower.image_tower.encoder.layers.7.mlp.fc1.weight']\n",
      "- This IS expected if you are initializing LlavaLlamaForCausalLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing LlavaLlamaForCausalLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:15<00:00, 15.02s/it]\n"
     ]
    }
   ],
   "source": [
    "from DPF.filters.videos.video_llava_filter import VideoLLaVAFilter\n",
    "\n",
    "datafilter = VideoLLaVAFilter(\n",
    "    prompt=\"detailed_video\",\n",
    "    device=\"cuda:0\",\n",
    "    workers=16,\n",
    "    batch_size=8\n",
    ")\n",
    "\n",
    "processor.apply_data_filter(datafilter)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c56bf304",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>video_path</th>\n",
       "      <th>text</th>\n",
       "      <th>split_name</th>\n",
       "      <th>is_correct</th>\n",
       "      <th>error</th>\n",
       "      <th>width</th>\n",
       "      <th>height</th>\n",
       "      <th>fps</th>\n",
       "      <th>duration</th>\n",
       "      <th>caption Video-LLaVA-7B prompt detailed_video</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>example_video_dataset/0/0.mp4</td>\n",
       "      <td>Businessman Rides In A Car</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>14.2800</td>\n",
       "      <td>This video is a black and white portrait of a ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>example_video_dataset/0/1.mp4</td>\n",
       "      <td>Workspace In Natural Light</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>10.0800</td>\n",
       "      <td>The video begins with a close-up shot of a tab...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>example_video_dataset/0/2.mp4</td>\n",
       "      <td>Woman In Dress Walking In Tulip Field</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>12.1200</td>\n",
       "      <td>The video starts with a woman walking through ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>example_video_dataset/0/3.mp4</td>\n",
       "      <td>Film Burns Overlay</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>23.976024</td>\n",
       "      <td>24.5245</td>\n",
       "      <td>This video features a woman with long blonde h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>example_video_dataset/0/4.mp4</td>\n",
       "      <td>Portrait Leader Of The Roman Army</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>29.970030</td>\n",
       "      <td>9.2092</td>\n",
       "      <td>The video showcases a man wearing a large, orn...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      video_path                                   text  \\\n",
       "0  example_video_dataset/0/0.mp4             Businessman Rides In A Car   \n",
       "1  example_video_dataset/0/1.mp4             Workspace In Natural Light   \n",
       "2  example_video_dataset/0/2.mp4  Woman In Dress Walking In Tulip Field   \n",
       "3  example_video_dataset/0/3.mp4                     Film Burns Overlay   \n",
       "4  example_video_dataset/0/4.mp4      Portrait Leader Of The Roman Army   \n",
       "\n",
       "  split_name  is_correct error  width  height        fps  duration  \\\n",
       "0          0        True  None   1280     720  25.000000   14.2800   \n",
       "1          0        True  None   1280     720  25.000000   10.0800   \n",
       "2          0        True  None   1280     720  25.000000   12.1200   \n",
       "3          0        True  None   1280     720  23.976024   24.5245   \n",
       "4          0        True  None   1280     720  29.970030    9.2092   \n",
       "\n",
       "        caption Video-LLaVA-7B prompt detailed_video  \n",
       "0  This video is a black and white portrait of a ...  \n",
       "1  The video begins with a close-up shot of a tab...  \n",
       "2  The video starts with a woman walking through ...  \n",
       "3  This video features a woman with long blonde h...  \n",
       "4  The video showcases a man wearing a large, orn...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processor.df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "16cce87a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"This video is a black and white portrait of a man sitting in the back of his car, dressed in formal attire. He is wearing a suit and tie, which is typical of the formal dress code. As he sits in his seat, he looks out of frame, indicating that he is either waiting for someone or simply enjoying the view outside. His gaze is directed towards the rearview mirror, suggesting that there might be something interesting happening outside the car. Overall, the video captures a moment of stillness and contemplation, with the man'taking his gazed fixed on something outside of view.\",\n",
       " 'The video begins with a close-up shot of a table with various items on it, including a laptop, a mouse, some books, flowers, vases, potted plants, cups, bowls, bottles, scissors, pens, an apple, notebooks, glasses, remote controls, candles and a cell phone. A wooden chair is also present in the scene. After the shot, the camera pans around the table, showing the items from different angles. Then, it cuts to a shot from a first-person perspective, with the person sitting at the desk and typing on the laptop. Throughout the video, there are various closeups of the objects on and around and in front of and behind the wooden table. Overall, this is a detailed and intricate video that showcases the different items and objects that are on a desks and tables, while also providing a glimpse into the daily activities of someone who uses the space.',\n",
       " \"The video starts with a woman walking through a field of colorful flowers. As she walks, she stops to admire the beauty of the flowers and takes a moment to enjoy the serene surroundings. She then proceeds to sit on a bench and continues to appreciate the vibrant colors and delicate petals of each flower. Her movements are graceful and deliberate, as if she is lost in the moment, completely immersed in nature.\\nThe camera captures the woman' s every movement, from her initial steps to her final resting place on the benches. Throughout the video, the camera lingers on each beautiful flower, allowing the viewer to take in its intricate details. Overall, this is a beautiful and peaceful video that showcases the simple joy of taking a walk through nature and appreciating the small things in life.\",\n",
       " 'This video features a woman with long blonde hair and a white shirt who is seen speaking to a camera. She is standing in front of a wall with a large painting of red and yellow hues. Throughout the video, the woman continues to speak to camera and is later joined by a man who appears in the frame. They both stand in various positions and locations, with the man also speaking directly to his own camera at one point.\\nThe video has a vibrant and energetic feel, thanks to its colorful backdrop and the dynamic movements of the individuals featured. Both the women and men are dressed in casual attire, which adds to this relaxed and informal atmosphere. Despite the lack of dialogue, their body language and facial expressions convey a sense of passion and enthusiasm for their subject matter. Overall, this is an engaging and visually appealing video that showcases the talents of its creators.',\n",
       " \"The video showcases a man wearing a large, ornate helmet and a blue cape. He appears to be a warrior or a knight, possibly from a historical period. In the video, he is seen walking towards the camera, looking off into the distance, as if he'd just arrived at his destination. As he approaches, his face becomes visible, revealing his stern expression. From his posture and the way he carries himself, it'a clear that he has been on a mission or on the move for some time.\\nThe video' style is a mix of natural lighting and dramatic shadows, creating a moody and atmospheric effect. It' a visually striking video that conveys a sense of strength and determination. Overall, the man' s presence and his attire create a powerful and captivating image that draws the viewer in.\"]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processor.df['caption Video-LLaVA-7B prompt detailed_video'].tolist()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "565dc7d9",
   "metadata": {},
   "source": [
    "## LLaVA 1.5 captioning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7ea2846f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Describe this video very shortly in 1-2 short sentences. Describe what is happening in this video.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d9730f5b5be43e7beffa3e09fe53095",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at openai/clip-vit-large-patch14-336 were not used when initializing CLIPVisionModel: ['text_model.encoder.layers.6.layer_norm2.weight', 'text_model.encoder.layers.6.mlp.fc1.bias', 'text_model.encoder.layers.3.self_attn.q_proj.weight', 'text_model.encoder.layers.0.layer_norm1.weight', 'text_model.encoder.layers.7.mlp.fc1.weight', 'text_model.encoder.layers.8.mlp.fc2.bias', 'text_model.encoder.layers.1.mlp.fc1.weight', 'text_model.encoder.layers.6.self_attn.q_proj.weight', 'text_model.encoder.layers.7.self_attn.q_proj.weight', 'text_model.encoder.layers.4.self_attn.v_proj.weight', 'text_model.encoder.layers.0.self_attn.v_proj.weight', 'text_model.encoder.layers.10.self_attn.v_proj.bias', 'text_model.encoder.layers.2.mlp.fc2.weight', 'text_model.encoder.layers.3.self_attn.k_proj.bias', 'text_model.encoder.layers.9.self_attn.k_proj.bias', 'text_model.encoder.layers.10.mlp.fc2.bias', 'text_model.encoder.layers.5.self_attn.q_proj.bias', 'text_model.encoder.layers.1.mlp.fc2.weight', 'text_model.encoder.layers.5.layer_norm1.bias', 'text_model.encoder.layers.4.mlp.fc2.bias', 'text_model.encoder.layers.7.self_attn.out_proj.weight', 'text_model.encoder.layers.1.layer_norm2.bias', 'text_model.encoder.layers.11.layer_norm2.weight', 'text_model.encoder.layers.8.mlp.fc1.weight', 'text_model.encoder.layers.11.layer_norm1.bias', 'text_model.encoder.layers.11.layer_norm1.weight', 'text_model.encoder.layers.4.self_attn.k_proj.bias', 'text_model.encoder.layers.2.layer_norm1.bias', 'text_model.encoder.layers.1.self_attn.q_proj.weight', 'text_model.encoder.layers.11.mlp.fc1.weight', 'text_model.encoder.layers.5.layer_norm2.weight', 'text_model.encoder.layers.3.mlp.fc1.weight', 'text_model.encoder.layers.1.layer_norm1.bias', 'text_model.encoder.layers.8.self_attn.k_proj.bias', 'text_model.encoder.layers.2.self_attn.out_proj.weight', 'text_model.encoder.layers.5.self_attn.v_proj.weight', 'text_model.encoder.layers.6.mlp.fc1.weight', 'visual_projection.weight', 'text_model.final_layer_norm.weight', 'text_model.encoder.layers.1.self_attn.k_proj.bias', 'text_model.encoder.layers.10.self_attn.q_proj.weight', 'text_model.encoder.layers.3.self_attn.v_proj.bias', 'text_model.encoder.layers.10.mlp.fc2.weight', 'text_model.encoder.layers.1.self_attn.q_proj.bias', 'text_model.encoder.layers.1.self_attn.v_proj.bias', 'text_model.encoder.layers.5.self_attn.v_proj.bias', 'text_model.encoder.layers.1.mlp.fc2.bias', 'text_model.encoder.layers.5.self_attn.out_proj.weight', 'text_model.encoder.layers.4.mlp.fc1.weight', 'text_model.encoder.layers.4.self_attn.k_proj.weight', 'text_model.encoder.layers.10.self_attn.k_proj.bias', 'text_model.encoder.layers.10.mlp.fc1.weight', 'text_model.encoder.layers.4.mlp.fc1.bias', 'text_model.encoder.layers.2.self_attn.q_proj.weight', 'text_model.embeddings.position_embedding.weight', 'text_model.encoder.layers.7.mlp.fc2.bias', 'text_model.encoder.layers.8.self_attn.v_proj.bias', 'logit_scale', 'text_model.encoder.layers.10.self_attn.v_proj.weight', 'text_model.encoder.layers.8.self_attn.out_proj.bias', 'text_model.encoder.layers.0.mlp.fc1.bias', 'text_model.encoder.layers.8.self_attn.v_proj.weight', 'text_model.encoder.layers.8.self_attn.k_proj.weight', 'text_model.encoder.layers.9.mlp.fc2.weight', 'text_model.encoder.layers.10.self_attn.k_proj.weight', 'text_projection.weight', 'text_model.final_layer_norm.bias', 'text_model.encoder.layers.0.mlp.fc2.weight', 'text_model.encoder.layers.3.mlp.fc1.bias', 'text_model.encoder.layers.11.self_attn.k_proj.bias', 'text_model.encoder.layers.1.self_attn.k_proj.weight', 'text_model.encoder.layers.10.layer_norm2.weight', 'text_model.encoder.layers.0.self_attn.q_proj.weight', 'text_model.encoder.layers.9.self_attn.q_proj.weight', 'text_model.encoder.layers.11.self_attn.q_proj.weight', 'text_model.encoder.layers.8.layer_norm2.bias', 'text_model.encoder.layers.6.self_attn.v_proj.bias', 'text_model.encoder.layers.9.self_attn.out_proj.weight', 'text_model.encoder.layers.4.self_attn.out_proj.bias', 'text_model.encoder.layers.3.self_attn.v_proj.weight', 'text_model.encoder.layers.7.layer_norm2.weight', 'text_model.encoder.layers.2.self_attn.v_proj.weight', 'text_model.encoder.layers.8.layer_norm2.weight', 'text_model.encoder.layers.3.layer_norm1.weight', 'text_model.encoder.layers.7.layer_norm1.weight', 'text_model.encoder.layers.6.mlp.fc2.weight', 'text_model.encoder.layers.4.self_attn.q_proj.bias', 'text_model.encoder.layers.6.self_attn.out_proj.bias', 'text_model.encoder.layers.7.self_attn.k_proj.weight', 'text_model.encoder.layers.7.self_attn.out_proj.bias', 'text_model.encoder.layers.10.self_attn.out_proj.bias', 'text_model.encoder.layers.3.self_attn.out_proj.bias', 'text_model.encoder.layers.0.self_attn.k_proj.weight', 'text_model.encoder.layers.3.mlp.fc2.weight', 'text_model.encoder.layers.4.self_attn.v_proj.bias', 'text_model.encoder.layers.1.self_attn.out_proj.weight', 'text_model.encoder.layers.2.self_attn.q_proj.bias', 'text_model.encoder.layers.4.layer_norm2.bias', 'text_model.encoder.layers.11.mlp.fc2.bias', 'text_model.encoder.layers.3.layer_norm1.bias', 'text_model.encoder.layers.3.mlp.fc2.bias', 'text_model.encoder.layers.6.self_attn.out_proj.weight', 'text_model.encoder.layers.0.layer_norm2.bias', 'text_model.encoder.layers.6.layer_norm2.bias', 'text_model.encoder.layers.1.self_attn.out_proj.bias', 'text_model.encoder.layers.3.self_attn.k_proj.weight', 'text_model.encoder.layers.8.layer_norm1.weight', 'text_model.encoder.layers.5.mlp.fc2.weight', 'text_model.encoder.layers.11.self_attn.out_proj.weight', 'text_model.encoder.layers.0.layer_norm1.bias', 'text_model.encoder.layers.9.mlp.fc1.bias', 'text_model.encoder.layers.9.mlp.fc2.bias', 'text_model.encoder.layers.6.layer_norm1.weight', 'text_model.encoder.layers.3.layer_norm2.bias', 'text_model.encoder.layers.10.self_attn.out_proj.weight', 'text_model.encoder.layers.5.self_attn.q_proj.weight', 'text_model.encoder.layers.9.mlp.fc1.weight', 'text_model.encoder.layers.9.layer_norm1.bias', 'text_model.encoder.layers.8.self_attn.out_proj.weight', 'text_model.encoder.layers.2.layer_norm2.weight', 'text_model.encoder.layers.0.mlp.fc1.weight', 'text_model.encoder.layers.2.layer_norm2.bias', 'text_model.encoder.layers.2.layer_norm1.weight', 'text_model.encoder.layers.7.mlp.fc1.bias', 'text_model.encoder.layers.3.self_attn.q_proj.bias', 'text_model.encoder.layers.2.self_attn.v_proj.bias', 'text_model.encoder.layers.11.layer_norm2.bias', 'text_model.encoder.layers.8.mlp.fc1.bias', 'text_model.encoder.layers.11.mlp.fc1.bias', 'text_model.encoder.layers.7.layer_norm2.bias', 'text_model.encoder.layers.8.self_attn.q_proj.bias', 'text_model.encoder.layers.6.self_attn.k_proj.bias', 'text_model.encoder.layers.0.self_attn.k_proj.bias', 'text_model.encoder.layers.11.mlp.fc2.weight', 'text_model.encoder.layers.8.mlp.fc2.weight', 'text_model.encoder.layers.9.self_attn.out_proj.bias', 'text_model.encoder.layers.4.self_attn.out_proj.weight', 'text_model.encoder.layers.5.mlp.fc1.bias', 'text_model.encoder.layers.6.mlp.fc2.bias', 'text_model.encoder.layers.0.self_attn.out_proj.bias', 'text_model.encoder.layers.5.self_attn.out_proj.bias', 'text_model.encoder.layers.1.layer_norm1.weight', 'text_model.embeddings.token_embedding.weight', 'text_model.encoder.layers.4.self_attn.q_proj.weight', 'text_model.encoder.layers.5.mlp.fc2.bias', 'text_model.encoder.layers.9.layer_norm2.bias', 'text_model.encoder.layers.4.layer_norm2.weight', 'text_model.encoder.layers.11.self_attn.out_proj.bias', 'text_model.encoder.layers.9.layer_norm1.weight', 'text_model.encoder.layers.10.layer_norm1.bias', 'text_model.encoder.layers.5.self_attn.k_proj.bias', 'text_model.encoder.layers.5.layer_norm2.bias', 'text_model.encoder.layers.0.self_attn.v_proj.bias', 'text_model.encoder.layers.4.layer_norm1.bias', 'text_model.encoder.layers.9.self_attn.q_proj.bias', 'text_model.encoder.layers.3.self_attn.out_proj.weight', 'text_model.encoder.layers.7.layer_norm1.bias', 'text_model.encoder.layers.0.layer_norm2.weight', 'text_model.encoder.layers.2.mlp.fc1.bias', 'text_model.encoder.layers.5.self_attn.k_proj.weight', 'text_model.encoder.layers.3.layer_norm2.weight', 'text_model.encoder.layers.5.layer_norm1.weight', 'text_model.encoder.layers.0.mlp.fc2.bias', 'text_model.encoder.layers.1.mlp.fc1.bias', 'text_model.encoder.layers.6.self_attn.q_proj.bias', 'text_model.encoder.layers.7.self_attn.k_proj.bias', 'text_model.encoder.layers.1.self_attn.v_proj.weight', 'text_model.encoder.layers.2.mlp.fc1.weight', 'text_model.encoder.layers.11.self_attn.v_proj.weight', 'text_model.encoder.layers.10.self_attn.q_proj.bias', 'text_model.encoder.layers.4.mlp.fc2.weight', 'text_model.encoder.layers.0.self_attn.q_proj.bias', 'text_model.encoder.layers.5.mlp.fc1.weight', 'text_model.encoder.layers.10.layer_norm1.weight', 'text_model.encoder.layers.9.layer_norm2.weight', 'text_model.encoder.layers.8.self_attn.q_proj.weight', 'text_model.encoder.layers.7.self_attn.v_proj.weight', 'text_model.encoder.layers.8.layer_norm1.bias', 'text_model.encoder.layers.4.layer_norm1.weight', 'text_model.encoder.layers.9.self_attn.v_proj.bias', 'text_model.encoder.layers.10.mlp.fc1.bias', 'text_model.encoder.layers.9.self_attn.k_proj.weight', 'text_model.encoder.layers.6.layer_norm1.bias', 'text_model.encoder.layers.6.self_attn.k_proj.weight', 'text_model.encoder.layers.7.self_attn.q_proj.bias', 'text_model.encoder.layers.9.self_attn.v_proj.weight', 'text_model.encoder.layers.6.self_attn.v_proj.weight', 'text_model.encoder.layers.2.self_attn.k_proj.weight', 'text_model.encoder.layers.11.self_attn.q_proj.bias', 'text_model.encoder.layers.11.self_attn.k_proj.weight', 'text_model.encoder.layers.1.layer_norm2.weight', 'text_model.encoder.layers.2.self_attn.k_proj.bias', 'text_model.encoder.layers.7.self_attn.v_proj.bias', 'text_model.embeddings.position_ids', 'text_model.encoder.layers.7.mlp.fc2.weight', 'text_model.encoder.layers.2.self_attn.out_proj.bias', 'text_model.encoder.layers.11.self_attn.v_proj.bias', 'text_model.encoder.layers.0.self_attn.out_proj.weight', 'text_model.encoder.layers.2.mlp.fc2.bias', 'text_model.encoder.layers.10.layer_norm2.bias']\n",
      "- This IS expected if you are initializing CLIPVisionModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing CLIPVisionModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [00:05<00:00,  1.12s/it]\n"
     ]
    }
   ],
   "source": [
    "from DPF.filters.images.llava_captioning_filter import LLaVaCaptioningFilter\n",
    "from DPF.filters.videos.image_filter_adapter import ImageFilterAdapter\n",
    "\n",
    "datafilter = LLaVaCaptioningFilter(workers=8, prompt='short-video', batch_size=1, device='cuda:0')\n",
    "video_adapter = ImageFilterAdapter(datafilter, 0.5, workers=8)\n",
    "\n",
    "processor.apply_data_filter(video_adapter)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6a839c0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>video_path</th>\n",
       "      <th>text</th>\n",
       "      <th>split_name</th>\n",
       "      <th>is_correct</th>\n",
       "      <th>error</th>\n",
       "      <th>width</th>\n",
       "      <th>height</th>\n",
       "      <th>fps</th>\n",
       "      <th>duration</th>\n",
       "      <th>mean_optical_flow_farneback</th>\n",
       "      <th>mean_optical_flow_raft</th>\n",
       "      <th>caption liuhaotian/llava-v1.5-13b prompt short-video</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>example_video_dataset/0/0.mp4</td>\n",
       "      <td>Businessman Rides In A Car</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>14.2800</td>\n",
       "      <td>7.428</td>\n",
       "      <td>20.997999</td>\n",
       "      <td>A man in a suit is sitting in a car, looking o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>example_video_dataset/0/1.mp4</td>\n",
       "      <td>Workspace In Natural Light</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>10.0800</td>\n",
       "      <td>5.926</td>\n",
       "      <td>38.886002</td>\n",
       "      <td>The image features a white desk with a laptop,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>example_video_dataset/0/2.mp4</td>\n",
       "      <td>Woman In Dress Walking In Tulip Field</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>12.1200</td>\n",
       "      <td>10.874</td>\n",
       "      <td>49.102001</td>\n",
       "      <td>A woman is walking through a flower garden, ca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>example_video_dataset/0/3.mp4</td>\n",
       "      <td>Film Burns Overlay</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>23.976024</td>\n",
       "      <td>24.5245</td>\n",
       "      <td>0.194</td>\n",
       "      <td>169.417999</td>\n",
       "      <td>The video is a colorful, abstract scene with a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>example_video_dataset/0/4.mp4</td>\n",
       "      <td>Portrait Leader Of The Roman Army</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>1280</td>\n",
       "      <td>720</td>\n",
       "      <td>29.970030</td>\n",
       "      <td>9.2092</td>\n",
       "      <td>10.816</td>\n",
       "      <td>65.942001</td>\n",
       "      <td>A man wearing a helmet and a blue feather is s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      video_path                                   text  \\\n",
       "0  example_video_dataset/0/0.mp4             Businessman Rides In A Car   \n",
       "1  example_video_dataset/0/1.mp4             Workspace In Natural Light   \n",
       "2  example_video_dataset/0/2.mp4  Woman In Dress Walking In Tulip Field   \n",
       "3  example_video_dataset/0/3.mp4                     Film Burns Overlay   \n",
       "4  example_video_dataset/0/4.mp4      Portrait Leader Of The Roman Army   \n",
       "\n",
       "  split_name  is_correct error  width  height        fps  duration  \\\n",
       "0          0        True  None   1280     720  25.000000   14.2800   \n",
       "1          0        True  None   1280     720  25.000000   10.0800   \n",
       "2          0        True  None   1280     720  25.000000   12.1200   \n",
       "3          0        True  None   1280     720  23.976024   24.5245   \n",
       "4          0        True  None   1280     720  29.970030    9.2092   \n",
       "\n",
       "   mean_optical_flow_farneback  mean_optical_flow_raft  \\\n",
       "0                        7.428               20.997999   \n",
       "1                        5.926               38.886002   \n",
       "2                       10.874               49.102001   \n",
       "3                        0.194              169.417999   \n",
       "4                       10.816               65.942001   \n",
       "\n",
       "  caption liuhaotian/llava-v1.5-13b prompt short-video  \n",
       "0  A man in a suit is sitting in a car, looking o...    \n",
       "1  The image features a white desk with a laptop,...    \n",
       "2  A woman is walking through a flower garden, ca...    \n",
       "3  The video is a colorful, abstract scene with a...    \n",
       "4  A man wearing a helmet and a blue feather is s...    "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processor.df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "541c981b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['A man in a suit is sitting in a car, looking out the window.',\n",
       " 'The image features a white desk with a laptop, a chair, and a vase. The desk is situated in front of a window, and the laptop is open, possibly in use.',\n",
       " 'A woman is walking through a flower garden, carrying a hat.',\n",
       " 'The video is a colorful, abstract scene with a blurry background and a bright yellow light in the foreground.',\n",
       " 'A man wearing a helmet and a blue feather is standing in front of a blue sky.']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processor.df['caption liuhaotian/llava-v1.5-13b prompt short-video'].tolist()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fba4f11a",
   "metadata": {},
   "source": [
    "## LITA captioning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b8432fe5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2024-05-22 00:27:18,567] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using the legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This means that tokens that come after special tokens will not be properly handled. We recommend you to read the related pull request available at https://github.com/huggingface/transformers/pull/24565\n",
      "/home/user/conda/envs/dpf/lib/python3.11/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "Loading checkpoint shards: 100%|██████████| 3/3 [00:20<00:00,  6.91s/it]\n",
      "Some weights of the model checkpoint at ./lita-vicuna-v1-3-13b-finetune were not used when initializing LitaLlamaForCausalLM: ['model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.post_layernorm.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.embeddings.class_embedding', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.post_layernorm.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.pre_layrnorm.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.pre_layrnorm.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.embeddings.position_embedding.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight']\n",
      "- This IS expected if you are initializing LitaLlamaForCausalLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing LitaLlamaForCausalLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "100%|██████████| 1/1 [00:16<00:00, 16.25s/it]\n"
     ]
    }
   ],
   "source": [
    "from DPF.filters.videos.lita_filter import LITAFilter\n",
    "\n",
    "datafilter = LITAFilter(batch_size=8)\n",
    "\n",
    "processor.apply_data_filter(datafilter)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "03a039dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['caption lita-vicuna-v1-3-13b-finetune prompt detailed_video']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datafilter.result_columns\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3c6ef8f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"The image features a man wearing a suit and tie, sitting in a car and looking out the window. The scene appears to be set in the rain, as the man is seen through the rain-covered window. The man's attire and the car's interior suggest a professional or formal setting. The overall atmosphere of the image is somewhat mysterious and intriguing, as the man's gaze seems to be directed towards something beyond the viewer's perspective.\",\n",
       " 'The video is a white room setup with a desk and chair, featuring a laptop, keyboard, mouse, and a potted plant. The room appears to be dimly lit, giving it a cozy atmosphere. The focus of the video is a laptop screen, which is displaying a video of a woman talking. The setup seems to be staged for a video call or a virtual meeting. The room contains a few other items, such as a cup, a vase, and a remote, but they are not the main subjects of the video.',\n",
       " 'The video is a short, whimsical scene featuring a woman walking through a field of flowers with a large vase on her head. The field is filled with numerous flowers of various colors, creating a vibrant and lively atmosphere. The woman appears to be enjoying her walk, and the vase on her head adds a touch of humor to the scene. The overall style of the video can be described as playful and lighthearted, evoking a sense of joy and wonder.',\n",
       " 'The video is a psychedelic, colorful, and trippy animation that features a logo and the number \"2\". The logo is a combination of letters and numbers, and it is displayed in various colors and patterns throughout the animation. The video has a hypnotic and mind-bending effect, with the logo and number 2 constantly changing and moving in a loop. This unique and artistic style makes the video stand out and leaves a lasting impression on the viewer.',\n",
       " 'The video features a man dressed in a metal suit, complete with a helmet, holding a sword. He is standing in the middle of a field, possibly on a movie set, as he is wearing a costume. The man appears to be staring into the distance, possibly preparing for a scene or just posing for a photo. The image gives off an epic and heroic vibe, as if the man is a warrior from ancient times.']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processor.df['caption lita-vicuna-v1-3-13b-finetune prompt detailed_video'].tolist()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1333d8a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dpf",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
